# -*- coding: utf-8 -*-
import requests
from bs4 import BeautifulSoup
import datetime
from mode import session,NEWS,Base,engine
import json
import logging
from rediscluster import StrictRedisCluster

startup_nodes = [{"host": "127.0.0.1", "port": "7000"}]
rc = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)

logging.basicConfig(level=logging.DEBUG,filename='rdaily_reptile.log',filemode='a',format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',datefmt='%a, %d %b %Y %H:%M:%S')
d=(datetime.date.today()-datetime.timedelta(days=-1)).strftime('%Y%m%d')
#当前时间
today=datetime.date.today()
url = 'https://news-at.zhihu.com/api/4/news/before/{}'.format(str(d))
#爬取的url
#url = 'https://news-at.zhihu.com/api/4/news/before/20181015' #补救措施
headers = {'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36'}
#浏览器标识
logging.info('开始爬取%s知乎日报数据' % today)
try:
    r = requests.get(url=url,headers=headers)
    logging.info('每日知乎日报数据获取成功')
except Exception as e:
    logging.warning("出错,错误原因 %s" % e)

try:
    date = datetime.datetime.strptime(json.loads(r.text)['date'],'%Y%m%d').date()
    stories = json.loads(r.text)['stories']
except Exception as e:
    logging.warning("出错,错误原因 %s" % e)
newslist = []
n = 0
try:
    for s in stories:
        n+=1
        if 'images' in s.keys():
            ed_news=NEWS(id=s['id'],title=s['title'],images=s['images'][0],date=date)
        else:
            ed_news=NEWS(id=s['id'],title=s['title'],images='http://pic2.zhimg.com/5ab6d4684114d27e9dd5e6be2d7c5f31.jpg',date=date)
        rc.set(s['id'],'https://news-at.zhihu.com/api/4/news/{}'.format(str(s['id'])))
        newslist.append(ed_news)
    logging.info('已经将%s条日报写入redis,当前时间：%s' % (n,datetime.datetime.now()))
    session.add_all(newslist)
    session.commit()
    logging.info('已经将%s条日报写入数据库,当前时间：%s' % (n,datetime.datetime.now()))
except Exception as e:
    logging.warning("出错,错误原因 %s" % e)
